#!/usr/bin/perl -w
#Merging manuscripts in the system together into a unified manuscript

use strict;
use warnings;
use utf8;
#use encoding "utf8", STDIN => 'utf8', STDOUT => 'utf8';
use open ':encoding(utf8)';
use Algorithm::Diff qw(diff sdiff LCS traverse_sequences traverse_balanced);
use Unicode::Normalize qw(decompose NFC);
use lib '../';
use ImportHelpers;

#This is too greedy
#$ImportHelpers::dbh->do('DELETE FROM token_group_cluster_token');
#$ImportHelpers::dbh->do('DELETE FROM token_group_cluster');
#$ImportHelpers::dbh->do('DELETE FROM token_group');

#Delete the existing Unified Manuscript

my $umsID = ImportHelpers::insertWork(
	name      => "GNT Unified Manuscript",
	type      => 'unified-manuscript',
	language  => 'grc',
	osis_id   => 'GNTUMS',
	copyright => "Copyright 2009, <a href='http://openscriptures.org/'>Open Scriptures</a>",
	licenses  => [qw( http://creativecommons.org/licenses/by-sa/3.0/ )]
	#url => "urn:openscriptures:manuscript:unified"
);

	
print "\n";
print ("*" x 80 . "\n" );
print "* Merging works into Unified Manuscript\n";
print ("*" x 80 . "\n" ) ;

#Set the work.unified_work to $umsID
# @todo How are we going to know which manuscripts should be attempted for merging? We need to have a parent work or something
$ImportHelpers::dbh->do("UPDATE work SET unified_work = $umsID WHERE type IN ('manuscript','manuscript-edition') AND language = 'grc'");

#Get all of the manuscripts that are going to be merged into the unified manuscript
my $manuscriptQuery = $ImportHelpers::dbh->prepare("SELECT * FROM work WHERE type IN ('manuscript','manuscript-edition') AND unified_work = ?");
$manuscriptQuery->execute($umsID);

#TODO: We need to clear the previously set ums_token!!!
my $updateTokenUMSToken = $ImportHelpers::dbh->prepare("UPDATE token SET ums_token = ? WHERE id = ?");

# Given a set of Manuscripts {M1, M2, M3, ..., Mn}, each of which have a set of tokens with unique IDs
my @mss;
while(my $ms = $manuscriptQuery->fetchrow_hashref){
	push @mss, $ms;
}

#Iterate over all books in the New Testament
my $selectBookRef = $ImportHelpers::dbh->prepare("SELECT * FROM ref WHERE type = 'book' AND work = ?");
my $newTokenCount = 0;
foreach my $bookCode (@ImportHelpers::osisNewTestamentCodes){
	next if not ImportHelpers::isBookOK($bookCode);
	
	#print "\n";
	#print ("*" x 80 . "\n" ) ;
	#print "* $bookCode\n";
	#print ("*" x 80 . "\n" ) ;
	
	#Run a query getting all of the tokens from a work
	my $tokens = ImportHelpers::executeBookTokensQuery(book => $bookCode, work => $mss[0]->{id});
	next if not $tokens;
	
	#Normalize all tokens in M1 and insert them as the tokens for the UMS
	while(my $token = $tokens->fetchrow_hashref){
		Encode::_utf8_on( $token->{data} );
		ImportHelpers::insertToken(
			data => ImportHelpers::normalizeToken( $token->{data} ),
			type => $token->{type},
			position => $token->{position} + $newTokenCount, #NOTE: this position must be incremented by the number of new tokens inserted
			book => $bookCode,
			work => $umsID
		);
	}
	ImportHelpers::finishBookRefs($umsID);
	
	#Foreach of the MSS then compare with the tokens inserted into the UMS
	foreach my $ms (@mss){
		
		# Re-get all tokens from unified manuscript ($umsID)
		my(@umsTokens) = ();
		my $umsTokenQuery = ImportHelpers::executeBookTokensQuery(book => $bookCode, work => $umsID);
		my $umsTokenAtEnd; #When the merge is done, then the ->position of this will be subtracted from the position of the 
		while($umsTokenAtEnd = $umsTokenQuery->fetchrow_hashref){
			Encode::_utf8_on($umsTokenAtEnd->{data}); #recognize utf-8 from MySQL
			push @umsTokens, new Token( $umsTokenAtEnd );
		}
		
		# Get all tokens from the current manuscipt ($msID)
		my @msTokens = ();
		my $msTokenQuery = ImportHelpers::executeBookTokensQuery(book => $bookCode, work => $ms->{id});
		while(my $t = $msTokenQuery->fetchrow_hashref){
			Encode::_utf8_on($t->{data});
			push @msTokens, new Token( $t );
		}
		
		print "Merging " . $ms->{osis_id} . " into Unified MS: ";
		# 2. Diff all normalized tokens in $msID with $umsID, inserting any tokens from $msID not found in $umsID; insertion will increment the position of any subsequent tokens in $umsID
		my $umsTokenPosition = 0;
		my $umsTokenPositionOffset = 0;
		my $lastUmsBookTokenID;
		traverse_sequences(
			\@umsTokens,
			\@msTokens,
			{
				#When two tokens match, then we must create a new token group which consists of two clusters consisting of the two tokens
				MATCH => sub {
					my $a = shift;
					my $b = shift;
					$umsTokenPosition = $umsTokens[$a]->{position};
					
					#Instead of creating a whole token group, just assign the UMS token to be the ums_token of this token
					$updateTokenUMSToken->execute($umsTokens[$a]->{id}, $msTokens[$b]->{id});
					$lastUmsBookTokenID = $umsTokens[$a]->{id};
					
					#$insertTokenGroup->execute();
					#my $tokenGroupID = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					#
					#$insertTokenGroupcluster->execute($tokenGroupID, 0);
					#my $clusterID1 = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					#$insertTokenGroupclusterItem->execute($clusterID1, $umsTokens[$a]->{id});
					#my $clusterTokenID1 = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					#
					#$insertTokenGroupcluster->execute($tokenGroupID, 1);
					#my $clusterID2 = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					#$insertTokenGroupclusterItem->execute($clusterID2, $msTokens[$b]->{id});
					#my $clusterTokenID2 = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					print '.';
				},
				#If this function is called, then it just means the UMS has more tokens than the current MS, which is to be expected
				DISCARD_A => sub {
					my $a = shift;
					my $b = shift;
					$umsTokenPosition = $umsTokens[$a]->{position};
					$lastUmsBookTokenID = $umsTokens[$a]->{id};
					
					print '-';
				},
				#If this function is called, then it means there is a token in the current manuscript that doesn't exist in the unified Manuscript
				#  and we must insert it into the UMS immediately after the token previously examined
				DISCARD_B => sub {
					my $a = shift;
					my $b = shift;
					
					#$ImportHelpers::dbh->do("SET UNIQUE_CHECKS = 0");
					++$umsTokenPositionOffset;
					$ImportHelpers::dbh->do("UPDATE token SET position = position + 1 WHERE position >= $umsTokenPosition + $umsTokenPositionOffset AND work = $umsID");
					
					#Insert new token into UMS work
					$lastUmsBookTokenID = ImportHelpers::insertToken(
							data => $msTokens[$b]->{normalizedData},
							type => $msTokens[$b]->{type},
							position => $umsTokenPosition + $umsTokenPositionOffset,
							#book => $bookCode
					);
					$newTokenCount++;
					
					#Instead of creating a whole token group, just assign the UMS token to be the ums_token of this token
					$updateTokenUMSToken->execute($lastUmsBookTokenID, $msTokens[$b]->{id});
					
					#$insertTokenGroup->execute();
					#my $tokenGroupID = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					#
					#$insertTokenGroupcluster->execute($tokenGroupID, 0);
					#my $clusterID1 = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					#$insertTokenGroupclusterItem->execute($clusterID1, $tokenID);
					#my $clusterTokenID1 = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					#
					#$insertTokenGroupcluster->execute($tokenGroupID, 1);
					#my $clusterID2 = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					#$insertTokenGroupclusterItem->execute($clusterID2, $msTokens[$b]->{id});
					#my $clusterTokenID2 = $ImportHelpers::dbh->last_insert_id(undef, undef, undef, undef);
					
					print "+";
				}
			},
			#How each token is compared (what from each token is compared)
			sub {
				shift()->{normalizedData}
			}
		);
		
		#Now update the UMS work's book ref so that the end_token reflects
		
		#print "\nEND BOOK: ref=" . $ImportHelpers::workContext{$umsID}->{bookRefID} . ", umsToken=$lastUmsBookTokenID\n";
		#$ImportHelpers::updateRefEndToken->execute($lastUmsBookTokenID, $ImportHelpers::workContext{$umsID}->{bookRefID});
		#$ImportHelpers::workContext{$umsID}->{previousTokenID} = $lastUmsBookTokenID;
		$ImportHelpers::workContext{$umsID}->{tokenID} = $lastUmsBookTokenID; #This has to be set here, so that next book will use this as the end_token
		ImportHelpers::finishBookRefs($umsID);
		
		print "\n";
		
		# 3. For each pair of tokens in $msID and $umsID that are equivalent, insert a new token group consisting of the tokens from the two MSS as clusters
		
		
		# 4. Increment i, and go to #2
	}
	
}


# 1. If a unified manuscript Mu doesn't exist, then normalize all tokens in M1 and insert them as the tokens for Mu
#my $msTokens = $ImportHelpers::dbh->prepare("SELECT * FROM token WHERE work = ? ORDER BY position");

#SELECT position AS start_position FROM token INNER JOIN ref ON token.id = ref.start_token AND AND ref.type = 'book' WHERE ref.osis_id = ?
#SELECT position AS end_position   FROM token INNER JOIN ref ON token.id = ref.end_token   AND AND ref.type = 'book' WHERE ref.osis_id = ?
#SELECT token.* FROM token INNER JOIN ref ON ref.work = token.work WHERE token.work = ? AND ref.osis_id = ? AND token.position >=

=sql

SELECT token.*
FROM token
INNER JOIN (SELECT position AS start_position FROM token AS tokenStart WHERE tokenStart.id = ref.start_token)
INNER JOIN (SELECT position AS start_position FROM token AS tokenStart WHERE tokenStart.id = ref.start_token)

=cut

#my $selectStartPosition = $ImportHelpers::dbh->prepare("SELECT token.* FROM token INNER JOIN ref ON token.work = ref.work WHERE work = ? AND token.position >= ORDER BY position");
#my $msTokensForBook = $ImportHelpers::dbh->prepare("SELECT token.* FROM token INNER JOIN ref ON token.work = ref.work WHERE work = ? AND token.position >= ORDER BY position");

#my $insertToken = $ImportHelpers::dbh->prepare('INSERT INTO token (data, type, position, work, ums_token) VALUES(?,?,?,?,?)');

#my $insertTokenGroup = $ImportHelpers::dbh->prepare("INSERT INTO token_group (type) VALUES ('equivalent')");
#my $insertTokenGroupcluster = $ImportHelpers::dbh->prepare("INSERT INTO token_group_cluster (token_group, rel) VALUES (?, ?)");
#my $insertTokenGroupclusterItem = $ImportHelpers::dbh->prepare("INSERT INTO token_group_cluster_token (token_group_cluster, token) VALUES (?, ?)");


print "Done. Number of new tokens resulting from merge: $newTokenCount\n";

package Token;
sub new {
	my $package = shift;
	my $self = bless shift, $package;
	$self->{normalizedData} = ImportHelpers::normalizeToken($self->{data});
	return $self;
}

1;